import os, sys, requests
import time
try:
    import lxml
    from bs4 import BeautifulSoup
except:
    os.system("pip3 install bs4 lxml")


def fetchIndex(soup):
    return list(map(fetchIndex2, soup.findAll('a', {'class' : 'docsum-title'})))


def fetchIndex2(a):
    try:
        return a["href"].strip('/')
    except:
        return ""


def save2File():
    with open('output.pubmedContent.%s.txt' %(time.time()), mode='w', encoding='utf8') as f:
        for i in main.content.values():
            f.write(i)
 

def fetchAbsFromPubmedWithIndex(index):
    html = requests.get('https://pubmed.ncbi.nlm.nih.gov/%s/' %index, headers = main.headers, cookies = main.cookies)
    soup = BeautifulSoup(html.content, 'lxml')
    
    try:
        title = soup.find('h1', {'class' : 'heading-title'}).text.strip('\n \t')
        time = soup.find('button', {'id' : 'full-view-journal-trigger'}).text.strip('\n \t')
        for i in soup.find('div', {'class' : 'article-citation'}).findChildren('span'):
            time += i.text.strip('\n \t')
        abstract = ''
        for i in soup.find('div', {'id' : 'enc-abstract'}).findChildren('p'):
            abstract = '%s\t%s\n' %(abstract, i.text.strip(' \t\n').replace('\n', ''))

        main.content[index] = '%s\n%s\n%s\n\n%s\n\n\n\n\n' %(index, title, time, abstract)
    except:
        with open('log.pubmed.txt', mode='a+', encoding='utf8') as f:
            f.write("Fetching Abstract: Reaching https://pubmed.ncbi.nlm.nih.gov/%s/ Error\n" %(index))
    return


def main(url = "", index = []):
    main.index = []
    main.content = {}
    main.headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15' }
    main.cookies = {'ncbi_sid' : '9344DD051C70E983_26432SID', 'WebEnv' : '1D116a%409344DD051C70E983_26432SID', 'ncbi_prevPHID' : 'CE8A176A1CF9E8510000000000630062', 'MyNcbiSigninPreferences' : 'O21zbGl2ZSY%3D', 'WebCubbyUser' : 'VCHEHEMYCTPI114TBLZVUZLT217ZZZA1%3Blogged-in%3Dtrue%3Bmy-name%3D2907821379%2540qq.com%3Bpersistent%3Dtrue%409344DD051C70E983_26432SID', 'NIHSMSESSION' : '0mRJr30HIQFdarUCkyUqzJc0ilsnUzZiao+niuo6/rhb6vr3paDaqwC5jt2XA+07EIlbBHdhCDOfK8JDbfd6P7RTqY6PsmOiVNDgGB1Aiq60Gn593z72IjNSHDtHm4jisARx84iUTG9rdGJZPZEduZvIfxHQCHkPWYbp/EsF8xSbCPN050E8B7NsXRwub7CecehZANqzEycVk9BvpzdxzdggnjADVs9OXPVWaCB0N0mXcIoIe0colRWXnxXUzks+NM0R+G3Y3s5W4UWAy4U71rEVh9O0JqDFqP1W/oJ6IgNmnteQ02rRrqaoTuME5zF5R5+IOl1dqxBO5E5oB7YRQ3TueLYrX8bvUbC68exwgPlz5zQhNhDNRj042m7jnIWzHgLU38yXBsjMID3lWDD0E6mtBw+k7S1ZKtHp02aGEE0cDssqAsJPhzI+OjMeUQgJDxti62Ul5Jof+cUZekXMr7Jqav22mBaG9gQgU/o7m0mTsS4SnVI6TE0RF04VV2bVbDLhqM+mEHlhwkizj5Sfw4MxdgAAtqOxedxTMB5kjagAM7TKzquhDoBqHNsIytSchEVYI2WTpzD9MFpzkTfbfaNxDRD+0KViFQH6m0zV6OUxRFRFXXMpB92aYl/f9d4zBdQs1niBJTa6e/Rjz/gV2eGTOvevztDA8yUkuN8avNrwzjCYeMri5KiepyT10ugH41ACAg7RQm+mI3KcZ1hYC2t5PW4D6RNkERKM+YsuVJKNpBxfZe2jR+f3qweiCeZXbJPSxvLbGqPbSqb2DEIVUCJPOZ6ePt/JJb377MxDLmO3sWc8VOA8JcXu9Qn3pYL5AziS2WahWbpi/D8Umz8JTMaKXGxq0mDfZz/4FF3UOAaSQMx7SiNMuX0DPg4fIteQ4ttOkB/VvHl4BJhywNo6gZYH0dFHp+iVkEioHRJjajWd+hg8l6udDmdyN2xzwy1DIO8/XkJ7lNaUh/gAfnphxAFViiWIpNF5D+F2fwYKnC8bZBJlAgF/XlqXLNm+ozwy03FQEpMruE76LrDkS5CMmaHxvIEcr+xZHOE6GBtNhx6tbSEyzGdRcg2YdKagJJg7iLGVR5VSD4dh54l/57iatIDhUP4GEO8DVco3/2033P23apNMzWKGZ76JjBqhx3KO9FBieYQryjS+7kBKJLaN7C1BceQXXYVLEzHQ1jmgCRSW8untXpygkWUvM8hlYaBMa/9mGdFY5LY5ljjNo9IYCgzVkFL3oe5a7L6Ry/n+9yfrlSgGG+64gc/DB8PHdMejsnWOuL+cEH0eXEI1Ud2gAtgEI+o/jGTO', 'NIHSMPROFILE' : '5O6L%2BikW%2FcwWawwV%2F6ixAagzjIMcwzz5fNGkaFy2NiYwBTRPkOHfjDZdH4%2FEja4prkM%2Fi9Iw3tDC2a1ar1lL1sRZrZ%2FKzgDIYJopbFx4ukCpK1KbrFh6j74KrRBZTyyU9xrGQQm1GicU%2FwFlihqR%2Bz7cnbHUQMScOGJCIBff8oni%2Beu5sgrIs%2FMvFAzZ%2FlRLucPnNOpjVyyOPgW%2BTzQkb4BbeBkQfGl%2BFPeRUL5I39t3s89Sg9CUyhXNa19Xck3WZFFT%2B9nmzYpLigoGMsSRodMIYq05uOw0%2BPEq9E75mPip3eGCR2mrn0GUucoaCx1ILygsllsbSNxZuXCxAKvlO1HQwNwPRTCuL%2FrCelYu2WtWwdQ3zqi2WRRMwp4CWFR%2F%2FairgUktIkKNWxIuOBoiZQ%3D%3D', 'PRESERVTARGET' : '%2FtYTXpgzJne16bwfb4ZN2lGInyYoZNk58TVbSvhIR0njSJplCp65%2BiF2SZAktvmmznDxgJBJhBCH%0ANoo2je1cMk0RXykLSXa4UwW7u0%2B%2Fc1X7WzHdCi209NjSVDPLNfOmFzmtz50Uuh6EfD95OQ%2BYQ2B%2B%0Aq7BP3es9s8ArLlZd9XW7NS72Ulu8cigULF%2FZADnu%2FPZf8DmPLOXuV6xWf0fqcNlZXwWhiCjrPJiU%0AU594rDm20QBWFe5y0VjWXnJtzYm7uSPkWDQYJ8htbKyWwjn4aG0xcYfTBSBUTOi9A%2Bo1BnUPHLIi%0A8V9%2Fi7S2i2vLCCwVTCSGS0pctKKWZRmzEmP9NB4rA167%2FSMuyX6ezHZNUyztiKaga84g5monl5bT%0AjNlmWeBFQV90piriK2wjmey3mIoTu2eJyDi%2Bx%2FO7pwMTfeiU2WXZ5h3U4kRBxw%2FR6%2FrCMYtVrzXp%0A%2FexiuMJDHQmiDPowP8dxw97tgs353jnBRGe8jpoCPoPG2hywQnwXtxW8SjWp19yTypxVFl4KnD1e%0A5aoPyq%2F7tPDRPbW7UikYuihFvX0mD1TH7A0G9Bk%2B36y%2F7jL8oW7OArzEbESjcx2aVRL%2B3VqzX1Oc%0AZcFWXfVarYgckE8EeyNwFwhPDoASs2T4SVNAJAQ38A0bYzCAxc6mQLqADqesOuuveClDDgB8WITg%0A1QnE32rGsLz37nzAQ89V', 'ncbi_pinger' : 'N4IgDgTgpgbg+mAFgSwCYgFwgMIDYCMAzAAyn4CcxAogCIBCdxArKaQEyWEcDsuAYvjrcAdPmEBbOIRAAaEAGMANsnkBrAHZQAHgBdMoYphCKAhgGcdAWkUB7AObJ1MmfJvqdUd5YDuEE2BlxAFdFHWRLGGQob0tkD3EZEyCdREsbMDC3MwACE3kwmChAkLCIqJjXdxNHKAgZMBM7KEtKj3cZWwd1Swam7ODQ8MjonsaoXOTUgDMbCHFRpoAdJ17m1urNCFkQfCMklLSM5CztwnIjABZyAA4L7YvDLG5r8ju5C7usJRUNbT13piXe64IxTEyKMxQe7cIzkJgw97XIxsXDSd7nLBTKCoWomTLdSA2SI4iCWABGyR0bhkQTMJnJlOpthMqEcdm2LCMiB0OjAZgwAHoBd4RcJ1PIycgxYpxGLkIhhHYiQKwLMdOCBWZkHZ1I5hFN5A4APzycSoAC8ADkJcgIQBlbW69QAMn2iDgaHN4jMykKzogcyg5uIzrMQQginN3N5ZgApIQAIKxth8ZN8EXeMU26Wy3UKpUwNN5VxBdxpzqOeSIKBqNPxvhkvKqODhxTxmjRvnJpiJ7spvt8MBBMnibFZyU5uX5okD53qVtRnld3v9lMZ8dS9QyqeKmcp4s2Us6cuOVQJ+Qlsv9wgNpstiPtztxtg9pMv1dMFNDkdj8UTre5vKu6Fu+fDOlMC5PvGb6pmuop/pu255sBRYXoeV6pjepqoO2Wo6o4VABrMHK7FgHJsEYAAy9jZI4uR2Bs2TeHEiDZAAsioEA2GYNhTDo2S+lAwiuOIIAAL5yKWzKoJouj6CAhDSJi4KQqcnwgGCEJQnIhBImROkYiALDXAiIAPEY3DkOQpkfEYUk2Cysn/GZgJYICchMCCWD4NcaJGTC3nuSABBGEQ+CAhJCg2OI4huE58kUVgphkmYcAHkepxGPcRgVuoHJZXIbCPDgBAkGQlC0AwzCsMQHDEFw1n8IIIhiJIflsKRxgmClaVoUeGCWgA8paVAYLlA3DVQ2xsIlIDpe4G6TshBbTYZFBFacxX4MQ1yGDpnXrXtCmzdtu33EpIDEMIhBiLsHkXYdHLqSwhhiWJQA', '_ga' : 'GA1.2.386308394.1640443282', '_gat' : '1', '_gat_dap' : '1', '_gid' : 'GA1.2.662421768.1640862746', '_gat_ncbiSg' : '1', 'pm-adjnav-sid' : '03Ai6pbeaMxl2TbvWlnyfA:516ebb1b0d4d6f8eaeb022cec213749b', 'pm-sid' : 'lXf3V_3WhZGe1XgLGx2cgw:516ebb1b0d4d6f8eaeb022cec213749b', 'pm-csrf' : 'IeZbshsJ651ikFQb6zYCr79YwsUaJBxd3SLHYK5lTOxfpkuzuZrIIaPMLzDLt9v6', 'pm-sessionid' : '8z6sdxn55lzgqw7nitvj4hwhw2cbi9ed', 'pmc.article.report' : '', 'ncbi-login-route' : 'mslive', 'pm_ncbi_alert_dismiss' : 'nCoV_shutdown' }
    if len(index) == 0:
        html = requests.get(url, headers = main.headers, cookies = main.cookies)
        soup = BeautifulSoup(html.content, 'lxml')
        main.index += fetchIndex(soup)

        totalPagesNum = int(soup.find('label', {'class' : 'of-total-pages'}).text[3:])
        for i in range(2, totalPagesNum+1):
            print("Page: \t%s / %s" %(i, totalPagesNum), end='\r')
            html = requests.get('%s&page=%s' %(url, i), headers = main.headers, cookies = main.cookies)
            soup = BeautifulSoup(html.content, 'lxml')
            main.index += fetchIndex(soup)
            time.sleep(0.2)
        
        print()
        print(main.index)
    else:
        main.index += index

    print("This process can be interrupt at any time.")
    for index, i in enumerate(main.index):
        try:
            print("Fetching Abstract: \t%s / %s" %(index+1, len(main.index)), end='\r')
            fetchAbsFromPubmedWithIndex(i)
            time.sleep(0.2)
        except:
            print()
            save2File()
            return

    print()
    save2File()


if __name__ == '__main__':
    if len(sys.argv) !=2:
        print("\tpython *this *url")
        sys.exit()
    
    main(
        url=sys.argv[1],
        index = []
        )

